{"cells":[{"metadata":{"id":"FE976FAA690442828878338D0C37EA03","collapsed":false},"cell_type":"code","outputs":[{"output_type":"execute_result","metadata":{},"data":{"text/plain":"Index(['label', 'title_sum', 'title_count', 'title_ctr',\n       'title_query_dot_similarity', 'title_query_wmd_dis', 'query_word_num',\n       'title_word_num', 'is_query_in_title', 'query_len', 'title_len',\n       'len_title-query', 'len_query/title', 'q_t_word_match_share',\n       'q_t_jaccard', 'q_t_common_words', 'q_t_total_unique_words',\n       'q_t_wc_diff', 'q_t_wc_ratio', 'q_t_wc_diff_unique',\n       'q_t_wc_ratio_unique', 'title_nunique_prefix', 'Levenshtein_dis',\n       'Levenshtein_ratio', 'Levenshtein_jaro', 'Levenshtein_jaro_winkler',\n       'longistStr_len', 'prefix_title_pos', 'prefix_title_relative_pos',\n       'len_title-query_word', 'len_query/title_word',\n       'title_query_leven_rate', 'lcseque_len', 'preifx_detail_title_pos',\n       'cityblock_sim', 'norm_similarity', 'cosine_similarity', '0_lsi',\n       '1_lsi', '2_lsi', '3_lsi', '4_lsi', '5_lsi', '6_lsi', '7_lsi', '8_lsi',\n       '9_lsi', '10_lsi', '11_lsi', '12_lsi', '13_lsi', '14_lsi', '15_lsi',\n       '16_lsi', '17_lsi', '18_lsi', '19_lsi'],\n      dtype='object')"},"execution_count":3}],"source":"train_data.columns","execution_count":3},{"metadata":{"id":"DA077636419D46818440A25C91A2CB53","mdEditEnable":false},"cell_type":"markdown","source":"# 下面是ctr特征生成"},{"metadata":{"id":"B27C9182087B4419868ED5C221AFDE42","mdEditEnable":false},"cell_type":"markdown","source":"### 思路：就是把用来做ctr的那9.5亿，分成16块，每一块分别groupby，会出来16个goupby的结果，\n### 然后把title设置成index，用DataFrame.add（DataFrame）这个函数，就可以按索引进行求和。"},{"metadata":{"id":"0535CCCAB10147559E993EAB889250A0"},"cell_type":"code","outputs":[],"source":"import multiprocessing\r\nfrom time import time\r\nimport pandas as pd\r\nimport gc\r\ntime1=time()\r\nquery_title_ctr=pd.DataFrame()\r\nquery_ctr=pd.DataFrame()\r\ntitle_ctr=pd.DataFrame()\r\n\r\n# data_path=\"/home/kesci/input/bytedance/first-round/\"\r\ndata_path=\"/home/kesci/input/bytedance/\"\r\ndata_store_path=\"first_zzp/\"\r\ntrain_path=data_path+\"train_final.csv\"\r\n\r\n\r\ndef eva_title_ctr(start_index,single_data_num):\r\n    part_data=pd.read_csv(train_path,header=None,skiprows=start_index,nrows=single_data_num,names=['query_id','query','query_title_id','title','label'],usecols=[1,3,4])\r\n    part_data['label']=part_data['label'].astype(int)\r\n    temp = part_data.groupby(['title'],as_index=False)['label'].agg({'title_sum':'sum','title_count':'count'})\r\n    temp['title_nunique_prefix'] = part_data.groupby(['title'],as_index=False)['query'].agg({'title_nunique_prefix':'nunique'})['title_nunique_prefix']\r\n    return [temp]\r\ndef getBayesSmoothParam(origion_rate):\r\n    origion_rate_mean = origion_rate.mean()\r\n    origion_rate_var = origion_rate.var()\r\n    alpha = origion_rate_mean / origion_rate_var * (origion_rate_mean * (1 - origion_rate_mean) - origion_rate_var)\r\n    beta = (1 - origion_rate_mean) / origion_rate_var * (origion_rate_mean * (1 - origion_rate_mean) - origion_rate_var)\r\n    return alpha, beta\r\n\r\n\r\ntitle_ctr_16=[]\r\npool = multiprocessing.Pool(processes=15)\r\n\r\n\r\n\r\n\r\n# for epoch,part_data in enumerate(pd.read_csv(train_path,header=None,chunksize=20000000,names=['query_id','query','query_title_id','title','label'],usecols=[1,3,4])):  \r\n###################说明\r\n###################range里的数，是你要读\r\ntime1=time()\r\nfor epoch in range(int(15)):    \r\n    single_data_num=int(450000000/15)\r\n    title_ctr_16.append(pool.apply_async(eva_title_ctr, [0+single_data_num*epoch,single_data_num]))\r\n    print(epoch+1)\r\n    \r\ntime2=time()\r\npool.close()\r\npool.join()\r\ntime3=time()\r\nprint(\"groupBy 用时间：\",time3-time1)\r\nfor index,single_three in enumerate(title_ctr_16):\r\n    time6=time()\r\n    gc.collect()\r\n    single_three=single_three.get()\r\n    title=single_three[0].set_index(['title'])\r\n    if title_ctr.empty:\r\n        title_ctr=title\r\n    else:\r\n        title_ctr=title_ctr.add(title,fill_value=0)\r\n    title_ctr_16[index]=None\r\n    time7=time()\r\n    print(\"第\",index,\"轮：\",(time7-time6)/60)\r\ntitle_ctr_16=[]\r\n\r\ntime4=time()\r\nprint((time4-time3)/60)\r\ntitle_ctr['title_sum']=title_ctr['title_sum'].astype(int)\r\ntitle_ctr['title_count']=title_ctr['title_count'].astype(int)\r\ntitle_ctr['title_nunique_prefix']=title_ctr['title_nunique_prefix'].astype(int)\r\ntitle_ctr['title_ctr']=title_ctr['title_sum']/title_ctr['title_count']\r\nalpha, beta = getBayesSmoothParam(title_ctr['title_ctr'])\r\ntitle_ctr['title_ctr'] = (title_ctr['title_sum'] + alpha) / (title_ctr['title_count'] + alpha + beta)\r\nte=pd.DataFrame(title_ctr)\r\ntitle_ctr=title_ctr.reset_index(drop=False)\r\n# title_ctr.to_pickle(data_store_path+\"title_ctr.pickle\")\r\ntitle_ctr.to_pickle(\"/home/kesci/work/first_zzp/ctr/title_ctr_feature_0-4.5.pickle\")\r\ntime5=time()\r\n\r\nprint((time5-time4)/60)","execution_count":null},{"metadata":{"id":"3E835FEB7ACB4CBF8A1C6198C3B802FE","mdEditEnable":false},"cell_type":"markdown","source":"# 下面是w2v特征生成\n### 此处说明：因为做特征的时间距离比赛结束还剩两天，所以为了防止出错，我没写for循环，而是一个一个cell盯着做的。"},{"metadata":{"id":"CDE0DC44A3E5471A89BF3C0B7E4C21B4","collapsed":false,"scrolled":true},"cell_type":"code","outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n","name":"stderr"},{"output_type":"stream","text":"start\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"import numpy as np\r\nimport pandas as pd\r\nimport time\r\nimport sys\r\nimport datetime\r\nimport gc\r\nfrom sklearn.model_selection import StratifiedShuffleSplit\r\nfrom sklearn.model_selection import KFold, cross_val_score, train_test_split\r\nfrom sklearn.model_selection import StratifiedKFold\r\nfrom sklearn.metrics import roc_auc_score, log_loss\r\nimport lightgbm as lgb\r\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\r\nfrom sklearn.feature_extraction.text import CountVectorizer\r\nfrom sklearn.feature_selection import chi2, SelectPercentile\r\nimport math\r\nfrom sklearn.metrics import f1_score\r\nimport jieba\r\nimport jieba.posseg as psg\r\nfrom collections import Counter\r\nimport functools\r\nfrom time import time\r\nfrom gensim.models import Word2Vec\r\nfrom time import sleep\r\nimport multiprocessing\r\nfrom time import time\r\nimport pandas as pd\r\nimport gc\r\nfrom gensim.models import Word2Vec\r\nimport pandas as pd\r\nimport numpy as np\r\nimport gensim\r\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\r\nfrom sklearn.decomposition import TruncatedSVD\r\nfrom scipy.stats import skew, kurtosis\r\nfrom scipy.spatial.distance import cosine, cityblock, jaccard, canberra, euclidean, minkowski, braycurtis\r\nfrom scipy.stats import skew, kurtosis\r\nimport gensim\r\nfrom gensim.models import Word2Vec\r\nfrom nltk import word_tokenize\r\nfrom nltk.corpus import stopwords\r\n# stop_words = stopwords.words('english')\r\nimport scipy\r\n\r\n\r\nw2v_model = Word2Vec.load('w2v_model2/w2v_all_data_model.txt')\r\nword_wv = w2v_model.wv\r\n\r\n\r\ndef get_w2v(train_data):\r\n    print(\"start\")\r\n    def get_w2v_feature(word_list):\r\n        w2v_size=100\r\n        word_list[0]=word_list[0].split(' ')\r\n        word_list[1]=word_list[1].split(' ')\r\n        import numpy as np\r\n        wmd_dist = w2v_model.wmdistance(word_list[0], word_list[1])\r\n        if wmd_dist==np.inf:\r\n            wmd_dist=np.nan\r\n        word_vectors_query = np.zeros((len(word_list[0]), w2v_size))\r\n        word_vectors_title = np.zeros((len(word_list[1]), w2v_size))\r\n        for i in range(len(word_list[0])):\r\n            if str(word_list[0][i]) in word_wv.vocab.keys():\r\n                word_vectors_query[i][:] = word_wv[str(word_list[0][i])]\r\n        for i in range(len(word_list[1])):\r\n            if str(word_list[1][i]) in word_wv.vocab.keys():\r\n                word_vectors_title[i][:] = word_wv[str(word_list[1][i])]\r\n        mean_array_query = np.mean(word_vectors_query, axis=0)\r\n        mean_array_title = np.mean(word_vectors_title, axis=0)\r\n        dot_similarity = np.dot(mean_array_title, mean_array_query)\r\n        \r\n\r\n        import numpy as np\r\n        return [wmd_dist,dot_similarity]\r\n    train_data = train_data.apply(lambda x : get_w2v_feature(x),axis=1,result_type='expand')\r\n    train_data.columns=['title_query_wmd_dis','title_query_dot_similarity']\r\n    for i in train_data:\r\n        if isinstance(train_data[i],np.float64):\r\n            train_data[i]=train_data[i].astype(np.float32)\r\n        if isinstance(train_data[i],np.int64):\r\n            train_data[i]=train_data[i].astype(np.int16)\r\n    print(\"w2v feature ended\")\r\n    return train_data\r\n\r\n\r\npool = multiprocessing.Pool(processes=10)\r\n# funcs=[get_w2v]\r\n# funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\r\n# funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\r\nall_features_contains_part_train=[]\r\n# train_data=pd.read_pickle(\"/home/kesci/work/first_zzp/train_data0-5.pickle\")\r\n# train_data=train_data[['query','title']]\r\ntrain_data=pd.read_pickle(\"/home/kesci/work/second_zzp/090-095.pickle\")\r\ntrain_data=train_data[['query','title']]\r\n# test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\r\n# train_data=pd.concat([train_data,train_data2,test_data],axis=0)\r\n# train_data=test_data\r\n# del train_data2\r\n# del test_data\r\n# import gc\r\n# gc.collect()\r\n\r\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\r\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\r\n\r\n\r\n\r\nglobal_epoch=0\r\nprint(\"start\")\r\nfea1='query'\r\nfea2='title'\r\n\r\n","execution_count":1},{"metadata":{"id":"E294F2D69EA74CD2AB526C53F82AB9F0","collapsed":false,"scrolled":true},"cell_type":"code","outputs":[{"output_type":"stream","text":"0\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=0+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_1.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":2},{"metadata":{"id":"2433E6BCBEF2451EA089ABC322C99B8E","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"1\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=5000000+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_2.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":3},{"metadata":{"id":"9A826653549C4B1784FBF46EFA3A30AC","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"2\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=10000000+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_3.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":4},{"metadata":{"id":"8E2AC43AC029475C84830EFA5798A365","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"3\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=15000000+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_4.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":5},{"metadata":{"id":"02D8CDBE366046A8856DE464F1FC8A8F","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"4\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=20000000+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_5.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":6},{"metadata":{"id":"538ED0084A38497DB1A03CAE1619E4EB","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"5\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=25000000+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_6.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":7},{"metadata":{"id":"3739D33727DB43A08BF824C5C5F19EB5","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"6\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=30000000+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_7.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":8},{"metadata":{"id":"8CE82207B5C04AD78E1DC4894B43CE2E","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"7\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=35000000+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_8.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":9},{"metadata":{"id":"8FD1AD65B75D47F889EABE4959F12FAB","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"8\nstart\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"start\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DeprecationWarning: Call to deprecated `wmdistance` (Method will be removed in 4.0.0, use self.wv.wmdistance() instead).\n","name":"stderr"},{"output_type":"stream","text":"w2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\nw2v feature ended\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=40000000+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_9.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":10},{"metadata":{"id":"87E23C75CC664C2284C59746AB0F447A","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"9\n","name":"stdout"}],"source":"funcs=[get_w2v]\r\nfor func in funcs:\r\n    print(global_epoch)\r\n    global_epoch+=1\r\n    all_train_data_size=5000000\r\n    cpu_size=10\r\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\r\n    one_feature=[]\r\n    for part_id in range(cpu_size):\r\n        start=45000000+part_id*single_part_of_train_size\r\n        end=start+single_part_of_train_size\r\n        single_part=train_data[start:end]\r\n        one_feature.append(pool.apply_async(func, [single_part]))\r\n    all_features_contains_part_train.append(one_feature)\r\n\r\npool.close()\r\npool.join()\r\nfor index,i in enumerate(all_features_contains_part_train):\r\n    single_contains_part=[]\r\n    for j in i:\r\n        single_contains_part.append(j.get())\r\n    all_features_contains_part_train[index]=single_contains_part\r\nall_features_contains_part_train=all_features_contains_part_train[0]\r\n\r\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\r\nall_features_contains_part_train.to_pickle(\"/home/kesci/train_w2v_fea/w2v_fea_10.pickle\")\r\nall_features_contains_part_train=[]\r\npool = multiprocessing.Pool(processes=10)","execution_count":11},{"metadata":{"id":"E98B055C409C4E1785443F94EF4E99E0","mdEditEnable":false},"cell_type":"markdown","source":"# 下面为一些fuzzy、query/title长度、最长公共子串之类的普通特征"},{"metadata":{"id":"E06B28F2804A40A581ACFA2977074C59","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[],"source":"import numpy as np\nimport pandas as pd\nimport time\nimport sys\nimport datetime\nimport gc\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.model_selection import KFold, cross_val_score, train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, log_loss\nimport lightgbm as lgb\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_selection import chi2, SelectPercentile\nimport math\nfrom sklearn.metrics import f1_score\nimport jieba\nimport jieba.posseg as psg\nfrom collections import Counter\nimport functools\nfrom time import time\nimport Levenshtein\nfrom gensim.models import Word2Vec\nfrom time import sleep\nimport multiprocessing\nfrom time import time\nimport pandas as pd\nimport gc\neooch_tongji=0\n\n###################################\ndef word_match_share(query,title):\n    q1words = {}\n    q2words = {}\n    for word in query:\n        q1words[word] = 1\n    for word in title:\n        q2words[word] = 1\n    if len(q1words) == 0 or len(q2words) == 0:\n        # The computer-generated chaff includes a few questions that are nothing but stopwords\n        return 0\n    shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]\n    shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]\n    R = (len(shared_words_in_q1) + len(shared_words_in_q2))/(len(q1words) + len(q2words))\n    return R\n\n\n\ndef jaccard(query,title):\n    wic = set(query).intersection(set(title))\n    uw = set(query).union(title)\n    if len(uw) == 0:\n        uw = [1]\n    return (len(wic) / len(uw))\n\n\n\ndef common_words(query,title):\n    return len(set(query).intersection(set(title)))\n\n\n\ndef total_unique_words(query,title):\n    return len(set(query).union(title))\n\n\n\ndef wc_diff(query,title):\n    return abs(len(query) - len(title))\n\n\n\ndef wc_ratio(query,title):\n    l1 = len(query)*1.0\n    l2 = len(title)\n    if l2 == 0:\n        return np.nan\n\n    if l1 / l2:\n        return l2 / l1\n\n    else:\n        return l1 / l2\n\n\n\ndef wc_diff_unique(query,title):\n    return abs(len(set(query)) - len(set(title)))\n\n\n\ndef wc_ratio_unique(query,title):\n    l1 = len(set(query)) * 1.0\n    l2 = len(set(title))\n    if l2 == 0:\n        return np.nan\n    if l1 / l2:\n        return l2 / l1\n    else:\n        return l1 / l2\n\n\n\n\n################\n\n\n\n\n\ndef findLongistSubstr(str1, str2):\n    '''\n    寻找最长公共子串\n    '''\n    dp = [([0] * len(str2)) for i in range(len(str1))]\n    maxlen = maxindex = 0\n    for i in range(0, len(str1)):\n        for j in range(0, len(str2)):\n            if str1[i] == str2[j]:\n                if i!=0 and j!=0:\n                    dp[i][j] = dp[i - 1][j - 1] + 1\n                if i == 0 or j == 0:\n                    dp[i][j] = 1\n                if dp[i][j] > maxlen:\n                    maxlen = dp[i][j]\n                    maxindex = i + 1 - maxlen\n    return str1[maxindex:maxindex + maxlen]\n    \ndef lcseque_lens(s1, s2): \n     # 生成字符串长度加1的0矩阵，m用来保存对应位置匹配的结果\n    m = [ [ 0 for x in range(len(s2)+1) ] for y in range(len(s1)+1) ] \n    # d用来记录转移方向\n    d = [ [ None for x in range(len(s2)+1) ] for y in range(len(s1)+1) ] \n    for p1 in range(len(s1)): \n        for p2 in range(len(s2)): \n            if s1[p1] == s2[p2]:            #字符匹配成功，则该位置的值为左上方的值加1\n                m[p1+1][p2+1] = m[p1][p2]+1\n                d[p1+1][p2+1] = 'ok'          \n            elif m[p1+1][p2] > m[p1][p2+1]:  #左值大于上值，则该位置的值为左值，并标记回溯时的方向\n                m[p1+1][p2+1] = m[p1+1][p2] \n                d[p1+1][p2+1] = 'left'          \n            else:                           #上值大于左值，则该位置的值为上值，并标记方向up\n                m[p1+1][p2+1] = m[p1][p2+1]   \n                d[p1+1][p2+1] = 'up'         \n    (p1, p2) = (len(s1), len(s2)) \n    s = [] \n    while m[p1][p2]:    #不为None时\n        c = d[p1][p2]\n        if c == 'ok':   #匹配成功，插入该字符，并向左上角找下一个\n            s.append(s1[p1-1])\n            p1 -= 1\n            p2 -= 1 \n        if c == 'left':  #根据标记，向左找下一个\n            p2 -= 1\n        if c == 'up':   #根据标记，向上找下一个\n            p1 -= 1\n    return len(s)\n\ndef get_prefix_loc_in_title(prefix,title):\n    if prefix not in title:\n        return -1\n    lens = len(prefix)\n    if prefix == title[:lens]:\n        return 0\n    elif prefix == title[-lens:]:\n        return 1\n    else:\n        return 2\n\ndef get_title_query_levenshtein_distance(df):\n    chinese_dict='事丸书万丑丕丽举其冀匈匙冯冲决冻况冷冶净凋凉凌准凑减凛卢卦厂历厅压厌厕厘厝厩厢厨厥切券剪劈刊刚列刘刓刎刖刨刬刭刺到刮刿剀刻刳刷剉剐剑剟剧剖剡剔剜副割剩剿剽党亏匜匡匣匽匪匿匮队阡邦防阶阱阬那阮邪阳阿陂陈邸阽附际邻陇陀邺阻阼陌陔郊郎陋限郁郑陛除陡郛郡郤险院陨部陲都郭陵陪陴陶陬隍隆隋随隈隅隘隔鄙鄘障隧隤隰酂兰关兹养凤却冈罔冕办劝务动助劲励努劬劭劾势勃勉勋勍勘勒勔募勤写冢击凿余亿仇仅仍代伋仞仪仗伧伥仿份伎价伉伦仳任伤似伟优伛伯伺但低佃伽估何伶你伾佗佚佣佑住侧侪侈佽供佳侥佼例侣侔侬侨使侍佻侠佯依侑侦俦促俄俘俭俊俚俪俩俜俟俏俗侮修俨俑倍俾倡俶倒俸候健借俱倨倦倔倥倪俳倩倏倭倚债值偿偾假傀偻偶偏停偷偎偕偃傲傍傧储傣傅傩傥催像儆僦僚僧僮僖僵僻儇儒千博产亩亲亵亶订讣讥认讧记讦讫让讱讪训议讹访讽讳讲诀论讴讼讻许讶诐词诋评诎识诉译诈诊证诅诧诚诞该诟诡话诨诘诓诔诠诗试详诩询诣诤诛诶诰诲诫诳诮说诵诱语谄调读诽课谅诺请谁谂谇诿谀诸谆诼诹谈谙谗谛谍谔谎谏谋谓谐谖谑谚谒谕谠谧谟谦谥谢谣谨谩谬谪谰谱谭谮谴谳谶廷发变叛叙艾节芒芃芋芭苍芳芙芥芦芡芮苏苇芜芽芸芷茇苞范苟苦茄茎苴苛苓茏茅茂茉苹苒苫苕苔英茔苑荜草茬茨荒荟荤荚荐茭荆荦茫茗茜荃荛荏茸荣茹荪荑荀药茵荫莞荷获莱莅莲莎荼莺莹莠菜萃菲菡菅菁菊菌萝萌萋菘萎萧营萦著菑菹葆葱蒂董葛葫蒋葵落葩葺葸萱蒉蒿蒺蓟蓝蒙蓦蓬蒲蓐蒜蓄蒸蔽蔻蔓蓰箫蔗蕉蕤蕊蔬蕴薄薨蕾薮藐藓薰藩藤藻蘖彻彷彼徂径征待很徊律徇徒徐徜徘徙循徯徭德徽巡边辽达过迈迄迁迆迂迟返还迒近违迕迓迎远这迨迪迭迩迥迫述迢逅迷逆适逃选逊逋逞递逗逢逦逑逡逝速通透途逍造逴逮逵逻逶逼遍遄道遁遏遑遒遂遗遐逾遇遨遘遛遣遥遭遮遴遵避遽邂邀邅邈邃导寺寿将失头夸奅奖奎奕奢奡奥干年巧巩弘弛张弧弥弩弭弯弹强弈弊广庇床庐庑序底店废府庙庞庖度庭庠座廊庾廋廒廓廉当录彝币布帏帐帛帘帔帖帜帙帮帧常帷帻幅帽幂幄幕幔幛幢幡叱叮句叫叶台叹召只吃吓吐吸吁吵呈吨含吼吝呕吮听吞吻吾呀吟呵咄呷咖咍呼咀呶咛咆呻味咏哀咤哈咳哄哗哪咷哇响哑咽咬哉咱咫咨啊唉哺唇哽哼唤唧唠哨唆唐哮唁哲唱啜啖啃售啴唾唯啸啄喳啻喘喋喊喝喙嗟喈喟喃喷啼嗢喧喑喁喻嗞嗷嗔嗤嗝嗑嗜嗣嗅嘎嘉嘘嗾嗽嘤嘲嘬嘿噎嘶嘻嘱噫噬嚎嚼囊驰驮驯驴驱驲骀驸驾驹驽驶驷驼驿驻驺骇骄骆骂骈骁骋骏验骖骐骑骗骚骛骘骜骞骤问闳闷闵闱闸阀阁闺阂闿闾闽闼闻阃阅阐阊阍阎阔阑阕阖阙阗宄宇宅宏完宝宠宛宙客宣宥宾宸宽宵宴寄寂密宿富寐寓寝察寨寮寰奶妈她妄妆妣妒妨妓妊姒妩妍妖妪姊姑姐姆妹妮始姹姜娇姣姱姥'\n    step=0\n    def dis(word_string):\n        query_len = len(word_string['query']) \n        title_len = len(word_string['title'])\n        len_title_jian_query = title_len - query_len\n        len_query_div_title = query_len / title_len\n        \n        word_set=set(word_string['query'].split()+word_string['title'].split())\n        word_dict={word:chinese_dict[index] for index,word in enumerate(word_set)}\n        query=''\n        title=''\n        for i in word_string['query'].split():\n            query+=word_dict[i]\n        for i in word_string['title'].split():\n            title+=word_dict[i]\n        query_word_num=len(query)\n        title_word_num=len(title)\n        len_title_jian_query_word=title_word_num-query_word_num\n        len_query_div_title_word=query_word_num/title_word_num\n        is_query_in_title=0\n        if query in title:\n            is_query_in_title=1\n        Levenshtein_dis=Levenshtein.distance(query, title)\n        Levenshtein_ratio=Levenshtein.ratio(query, title)\n        Levenshtein_jaro=Levenshtein.jaro(query, title)\n        Levenshtein_jaro_winkler=Levenshtein.jaro_winkler(query, title)\n        title_query_leven_rate=(Levenshtein_dis / (len(title) + 3))\n        longistStr_len=len(findLongistSubstr(word_string['query'],word_string['title']))\n        prefix_title_pos=title.find(query)\n        prefix_title_relative_pos=prefix_title_pos/len(title)\n        \n        list_query=list(query)\n        list_title=list(title)        \n        word_match_share2=word_match_share(list_query,list_title)\n        jaccard2=jaccard(list_query,list_title)\n        common_words2=common_words(list_query,list_title)\n        total_unique_words2=total_unique_words(list_query,list_title)\n        wc_diff2=wc_diff(list_query,list_title)\n        wc_ratio2=wc_ratio(list_query,list_title)\n        wc_diff_unique2=wc_diff_unique(list_query,list_title)\n        wc_ratio_unique2=wc_ratio_unique(list_query,list_title)\n        lcseque_len2=lcseque_lens(list_query,list_title)\n        preifx_detail_title_pos2=get_prefix_loc_in_title(list_query,list_title)\n        \n        return [query_len,title_len,len_title_jian_query,len_query_div_title,\n            query_word_num,title_word_num,len_title_jian_query_word,is_query_in_title,\n            len_query_div_title_word,Levenshtein_dis,Levenshtein_ratio,\n            Levenshtein_jaro,Levenshtein_jaro_winkler,title_query_leven_rate,\n            longistStr_len,prefix_title_pos,prefix_title_relative_pos,\n            word_match_share2,jaccard2,common_words2,total_unique_words2,wc_diff2,wc_ratio2,wc_diff_unique2,wc_ratio_unique2,\n            lcseque_len2,preifx_detail_title_pos2]\n    print(\"distance ended\")\n    temp_df=df[['query','title']].apply(dis,axis=1,result_type='expand')\n    temp_df.columns=['query_len','title_len','len_title-query','len_query/title',\n    'query_word_num','title_word_num','len_title-query_word','is_query_in_title',\n    'len_query/title_word','Levenshtein_dis','Levenshtein_ratio',\n    'Levenshtein_jaro','Levenshtein_jaro_winkler','title_query_leven_rate',\n    'longistStr_len','prefix_title_pos','prefix_title_relative_pos',\n     'q_t_word_match_share','q_t_jaccard','q_t_common_words','q_t_total_unique_words','q_t_wc_diff','q_t_wc_ratio'\n     ,'q_t_wc_diff_unique','q_t_wc_ratio_unique','lcseque_len','preifx_detail_title_pos']\n    return temp_df\n\n\n","execution_count":1},{"metadata":{"id":"9A6EF1BDB09849598671F453C0A86B63","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[],"source":"for i in range(20):\n    train_data[i*5000000:(i+1)*5000000].to_pickle(\"/home/kesci/test_finall_all_parts/part_\"+str(i)+\".pickle\")","execution_count":5},{"metadata":{"id":"A9872EF7F1AC43DBAE0AAB6587CC1EDA","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\ntrain_data=pd.read_pickle(\"/home/kesci/test_final_fea/test_data.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_1.pickle\")","execution_count":2},{"metadata":{"id":"DD2759F945F44D93BC44123D2B594670","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\n# train_data=pd.read_pickle(\"/home/kesci/blending/train_data5000w-6000w.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=10000000+part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_2.pickle\")","execution_count":3},{"metadata":{"id":"AB38D055CEBA43028CEB06EC207E9C91","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\n# train_data=pd.read_pickle(\"/home/kesci/blending/train_data5000w-6000w.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=20000000+part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_3.pickle\")","execution_count":4},{"metadata":{"id":"F1E80C346F7A4849A0473FCE1A60B252","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\n# train_data=pd.read_pickle(\"/home/kesci/blending/train_data5000w-6000w.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=30000000+part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_4.pickle\")","execution_count":5},{"metadata":{"id":"4559E211B19041DCB7A216956BE9F08A","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\n# train_data=pd.read_pickle(\"/home/kesci/blending/train_data5000w-6000w.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=40000000+part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_5.pickle\")","execution_count":6},{"metadata":{"id":"4A33FFD169454A2397B26C2A253BDFC4","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\n# train_data=pd.read_pickle(\"/home/kesci/blending/train_data5000w-6000w.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=50000000+part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_6.pickle\")","execution_count":7},{"metadata":{"id":"BABD916B1CC342A585D9E93386B453C5","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\n# train_data=pd.read_pickle(\"/home/kesci/blending/train_data5000w-6000w.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=60000000+part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_7.pickle\")","execution_count":8},{"metadata":{"id":"F55C93574C3047CA9816228A2187FD8C","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\n# train_data=pd.read_pickle(\"/home/kesci/blending/train_data5000w-6000w.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=70000000+part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_8.pickle\")","execution_count":9},{"metadata":{"id":"9A48C69F480E4A579BCA1CB5E914E97E","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\n# train_data=pd.read_pickle(\"/home/kesci/blending/train_data5000w-6000w.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=80000000+part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_9.pickle\")","execution_count":10},{"metadata":{"id":"262B98D5EDE547BF8765B762670F38E2","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[{"output_type":"stream","text":"start\n0\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\ndistance ended\n","name":"stdout"}],"source":"pool = multiprocessing.Pool(processes=10)\n# # funcs=[get_w2v]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance]\n# # funcs=[get_query_word_num,get_title_word_num,get_is_query_in_title_feature,get_string_len_feature,get_title_query_levenshtein_distance,get_w2v]\nall_features_contains_part_train=[]\n# train_data=pd.read_pickle(\"/home/kesci/blending/train_data5000w-6000w.pickle\")\n# # test_data=pd.read_pickle(\"/home/kesci/work/first_zzp/test_data.pickle\")\n# # test_data[['query']]=test_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['query']]=train_data[['query']].applymap(lambda x:x.replace('\\t',''))\n# # test_data[['title']]=test_data[['title']].applymap(lambda x:x.replace('\\t',''))\n# train_data[['title']]=train_data[['title']].applymap(lambda x:x.replace('\\t',''))\nglobal_epoch=0\nprint(\"start\")\n\n\nfea1='query'\nfea2='title'\nfuncs=[get_title_query_levenshtein_distance]\nfor func in funcs:\n    print(global_epoch)\n    global_epoch+=1\n    all_train_data_size=10000000\n    cpu_size=10\n    single_part_of_train_size=int(all_train_data_size/cpu_size)\n    one_feature=[]\n    for part_id in range(cpu_size):\n        start=90000000+part_id*single_part_of_train_size\n        end=start+single_part_of_train_size\n        single_part=train_data[start:end]\n        one_feature.append(pool.apply_async(func, [single_part]))\n    all_features_contains_part_train.append(one_feature)\n\npool.close()\npool.join()\nfor index,i in enumerate(all_features_contains_part_train):\n    single_contains_part=[]\n    for j in i:\n        single_contains_part.append(j.get())\n    all_features_contains_part_train[index]=single_contains_part\nall_features_contains_part_train=all_features_contains_part_train[0]\n\nall_features_contains_part_train=pd.concat(all_features_contains_part_train)\nall_features_contains_part_train.to_pickle(\"/home/kesci/blending/char_fea_10.pickle\")","execution_count":11},{"metadata":{"id":"B3C62EFCE4054B4E8A772B719763D0F3","mdEditEnable":false},"cell_type":"markdown","source":"# lsi特征（tsvd降维特征）"},{"metadata":{"id":"80AC2C26ADA3486D8023B19BCEF38D6E","mdEditEnable":false},"cell_type":"markdown","source":"## 下面是做词典"},{"metadata":{"id":"E9808BF66F6441B682B847E4354FB871"},"cell_type":"code","outputs":[],"source":"from time import time \r\nimport pandas as pd\r\nimport warnings\r\n\r\n# query=pd.read_pickle(\"/home/kesci/work/query_all.pickle\")\r\n# title=pd.read_pickle(\"/home/kesci/work/title_all.pickle\")\r\n\r\ntime1=time()\r\nwarnings.filterwarnings(action='ignore',category=UserWarning,module='gensim')\r\nfrom gensim import corpora,similarities,models\r\n# test_data_all=pd.read_pickle(\"query_all.pickle\")\r\nclass mycorpus(object):\r\n    def __iter__(self):\r\n        for index,doc in enumerate(test_data_all['query']):\r\n            if index%5000000==0:\r\n                print(index)\r\n            yield doc.split(' ')\r\ncorp = mycorpus()\r\n# dictionary.add_documents(corp)\r\ndictionary=corpora.Dictionary(corp)\r\ntime2=time()\r\nprint(time2-time1)","execution_count":null},{"metadata":{"id":"BBA329773DF54A2481250DCF4537CD90","mdEditEnable":false},"cell_type":"markdown","source":"## 下面是做tfidf"},{"metadata":{"id":"0F8D9F24B65E4459B1DFF326EE8446B8"},"cell_type":"code","outputs":[],"source":"from time import time \r\nimport warnings\r\ntime1=time()\r\nwarnings.filterwarnings(action='ignore',category=UserWarning,module='gensim')\r\nfrom gensim import corpora,similarities,models\r\nimport pandas as pd\r\n# test_data_all=pd.read_pickle(\"/home/kesci/work/query_all.pickle\")\r\n\r\nprint(111523155656)\r\nclass MyCorpus(object):\r\n    def __init__(self):\r\n        print('MyCorpus')\r\n    def __iter__(self):\r\n        for index,doc in enumerate(test_data_all['query']):\r\n            if index%5000000==0:\r\n                print(index)\r\n            yield dictionary.doc2bow(doc.split(' '))\r\ncorpus = MyCorpus()\r\n# corpus = [dictionary.doc2bow(text) for text in self.text]\r\ntfidf_model = models.TfidfModel(corpus, id2word=dictionary)\r\n# corpus_tfidf = tfidf_model[corpus]","execution_count":null},{"metadata":{"id":"03E9A63A3DF24A12A8662006831BE6F0","mdEditEnable":false},"cell_type":"markdown","source":"## 训练lsi模型（即tsvd模型）"},{"metadata":{"id":"6338D94AA6AB44CC89AD9C318CEB4BAB"},"cell_type":"code","outputs":[],"source":"from time import time \r\nimport warnings\r\ntime1=time()\r\nwarnings.filterwarnings(action='ignore',category=UserWarning,module='gensim')\r\nfrom gensim import corpora,similarities,models\r\nimport pandas as pd\r\n# test_data_all=pd.read_pickle(\"/home/kesci/work/query_all.pickle\")\r\ncorpus_tfidf = tfidf_model[corpus2]\r\nprint(111523155656)\r\nclass MyCorpus(object):\r\n    def __init__(self):\r\n        print('MyCorpus')\r\n    def __iter__(self):\r\n        for index,doc in enumerate(corpus_tfidf):\r\n            if index%2000000==0:\r\n                print(index)\r\n            yield doc\r\ncorpus3 = MyCorpus()\r\n# corpus = [dictionary.doc2bow(text) for text in self.text]\r\nlsi_model = models.LsiModel(corpus3, id2word=dictionary,chunksize=10000000,num_topics=20)\r\nlsi_model.save(\"lsi_model/lsi_all_query.lsi\")\r\ntime2=time()\r\nprint((time2-time1)/60)\r\n# corpus_tfidf = tfidf_model[corpus]","execution_count":null},{"metadata":{"id":"19B7FB4AEEB1470E83254AE7AB376F2A","mdEditEnable":false},"cell_type":"markdown","source":"## 使用lsi模型"},{"metadata":{"id":"E2D94581CC5247078948FA432DC7A86B","mdEditEnable":false},"cell_type":"markdown","source":"### 此处说明：将tfidf值传入lsi后，lsi返回的是一个迭代器，每次迭代返回一个list，需要用pd.DataFrame来将list转为DataFrame。 此处用了多进程，分批将list转为DataFrame。"},{"metadata":{"id":"E779C00C17484CA9A5B4FDA0F0FA4C35"},"cell_type":"code","outputs":[],"source":"from gensim import models\r\nlsi_model=models.LsiModel.load(\"/home/kesci/work/lsi_model/lsi_all_query.lsi\")\r\nimport pickle\r\nfw=open('dictionary/dictionary.pickle','rb')\r\ndictionary=pickle.load(fw)\r\nimport pickle\r\nfw=open('tfidf_model/tfidf_model.pickle','rb')\r\ntfidf_model=pickle.load(fw)\r\n\r\nimport multiprocessing\r\nimport pandas as pd\r\n# lsi_list=data1\r\n# del data1\r\n# import gc\r\n# gc.collect()\r\n\r\nlsi_columns=[str(i)+'_lsi' for i in range(20)]\r\ndef to_df(start,end,train_data):\r\n    print(\"started\")\r\n    lsi_list=[dictionary.doc2bow(text.split(' ')) for text in train_data['query'][start:end]]\r\n    lsi_list=lsi_model[tfidf_model[lsi_list]]\r\n    print(\"doc2bow\")\r\n    temp_df=pd.DataFrame(list(lsi_list),columns=lsi_columns)\r\n    print(\"apply_map\")\r\n    temp_df=temp_df.applymap(lambda x:x[1] if x!=None else np.nan)\r\n    print(\"ended\")\r\n    return temp_df\r\npool = multiprocessing.Pool(processes=16)\r\nsingle_data_num=int(5000000/16)\r\nall_data=[]","execution_count":null},{"metadata":{"id":"30EA4F48112B43128556D6DCECB4392A"},"cell_type":"code","outputs":[],"source":"for part_id in range(20):\r\n    pool = multiprocessing.Pool(processes=16)\r\n    all_data=[]\r\n    train_data=pd.read_pickle(\"/home/kesci/test_finall_all_parts/part_\"+str(part_id)+\".pickle\")\r\n    train_data=train_data.reset_index(drop=True)\r\n    print(\"start\")\r\n    for i in range(16):\r\n        start=0+i*single_data_num\r\n        end=start+single_data_num\r\n        all_data.append(pool.apply_async(to_df, [start,end,train_data]))\r\n    pool.close()\r\n    pool.join()\r\n    all_data=[single_part.get() for single_part in all_data]\r\n    all_data=pd.concat(all_data)\r\n    all_data=all_data.reset_index(drop=True)\r\n    # all_data_parts=[]\r\n    # all_data_parts.append(all_data)\r\n    all_data.to_pickle(\"/home/kesci/test_final_lsi/lsi_part_\"+str((part_id+1))+\".pickle\")","execution_count":null}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":0}